# coding: utf-8

import utils.my_utils as my_utils
import jieba
import config.setting as setting
import os

stop_words_set = None


def stop_words_filter(src_list):
    '''
    列表减法操作
    :param aList:
    :param bList:
    :return:
    '''
    global stop_words_set
    if stop_words_set is None:
        # log.info('start to load stop words')
        stop_words_set = set()
        for file_path in setting.NLP_STOP_WORD_FILE_LIST:
            if not os.path.exists(file_path):
                continue
            with open(file_path, 'rb') as f:
                lines = f.readlines()
                for line in lines:
                    stop_words_set.add(line.strip().decode())
    return [word for word in src_list
            if not (len(word) <= 1 or word.isdigit() or word in stop_words_set)]


def clean_and_seg(content):
    content = my_utils.clean_data(content)
    content = jieba.lcut(content, cut_all=False, HMM=False)
    return stop_words_filter(content)


def do(columns, all_data):
    '''
    进行nlp分词
    :param all_data: 输入
    :return: 输出一个list，list中每个单元是一个map，keys包括（id，origin_url，title，content）
    '''
    res_list = list()
    for record in all_data:
        item = dict()
        item['origin_id'] = record[columns.index('id')]
        item['content_id'] = record[columns.index('article_id')]
        item['origin_url'] = record[columns.index('origin_url')]
        # 保存清洗之前的title
        item['title'] = my_utils.clean_data_except_mark(record[columns.index('title')].decode())
        # 保存清洗之前的content
        item['content'] = my_utils.clean_data_except_mark(record[columns.index('content')].decode())
        # title 处理
        item['title_seg'] = clean_and_seg(record[columns.index('title')].decode())
        # content 处理
        item['content_seg'] = clean_and_seg(record[columns.index('content')].decode())
        # 获取发布时间
        item['publish_time'] = record[columns.index('publish_time')]
        # 获取记录创建时间
        item['create_time'] = record[columns.index('create_time')]
        # 获取记录更新时间
        item['update_time'] = record[columns.index('update_time')]
        res_list.append(item)
    return res_list
